# -*- coding: utf-8 -*-

import sys
import re
import json

from scrapy.spider import Spider
from bs4 import BeautifulSoup
from scrapy import Request
from scrapy import FormRequest
from urlparse import urljoin
from realEstateCommittee.commons import _REPLACE_NBSP
from realEstateCommittee.items import RealestatecommitteeItem

reload(sys)
sys.setdefaultencoding('utf8')


class RealEstateCommittee_ZS_LA_Spider(Spider):
    name = "RealEstateCommittee_ZS_LA"

    color_to_status = {"#F0FFB6": u"预售",
                       "#FFFF66": u"预定",
                       "#82B4FF": u"签约",
                       "#96E686": u"备案",
                       "#E5E5E5": u"自留",
                       "#FFCCFF": u"回迁",
                       "#CC3399": u"抵押",
                       "Salmon": u"政府回购",
                       "#FF0000": u"查封",
                       "#6666CC": u"限制",
                       "#7030A0": u"处理",
                       "#FF8D00": u"不可售",
                       "#CFE1E9": u"历史遗留",
                       "#646464": u"其他",
                       "#FFFFFF": u"地下室",
                       "#03E4BC": u"公厕"}

    PROJECT_URL = "project_url"
    PROJECT_TO_URL = "project_to_url"
    PROJECT_FLOOR_INFO_URL = "project_floor_info_url"
    PROJECT_PRESELL_URL = "project_presell_url"
    ALIAS = "alias"  # : "推广名称/别名",
    PROJECT = "project"  # :"项目名称"
    PROJECT_ADDR = "project_addr"  # : "项目地址",
    PRESALE_PERMIT = "presale_permit"  # :"预售证号码",
    BUILD_NO = "build_no"  # :"本期住宅楼号/幢号",
    RESIDENCE_SET = "residence_set"  # :"本期住宅总套数",
    RESIDENCE_AREA = "residence_area"  # :"本期住宅总面积",
    COMMERCIAL_AREA = "commercial_area"  # :"本期商业总面积",
    COMMERCIAL_SET = "commercial_set"  # :"本期商业总套数",
    ELSE_SET = "else_set"  # :"本期其他套数",
    ELSE_AREA = "else_area"  # :"本期其他面积",
    SOLD_RESIDENCE_SET = "sold_residence_set"  # :"本期已售住宅总套数",
    SOLD_RESIDENCE_AREA = "sold_residence_area"  # :"本期已售住宅总面积",
    SOLD_COMMERCIAL_SET = "sold_commercial_set"  # :"本期已售商业总套数",
    SOLD_COMMERCIAL_AREA = "sold_commercial_area"  # :"本期已售商业总面积",
    SOLD_ELSE_SET = "sold_else_set"  # :"本期已售其他套数",
    SOLD_ELSE_AREA = "sold_else_area"  # :"本期已售其他面积",
    UNSOLD_RESIDENCE_SET = "unsold_residence_set"  # :"本期未售住宅总套数",
    UNSOLD_RESIDENCE_AREA = "unsold_residence_area"  # :"本期未售住宅总面积",
    UNSOLD_COMMERCIAL_SET = "unsold_commercial_set"  # :"本期未售商业总套数",
    UNSOLD_COMMERCIAL_AREA = "unsold_commercial_area"  # :"本期未售商业总面积",
    UNSOLD_ELSE_SET = "unsold_else_set"  # :"本期未售其他套数",
    UNSOLD_ELSE_AREA = "unsold_else_area"  # :"本期未售其他面积",
    PROPERTY_TYPE = "property_type"  # :"物业类型/房屋类型",
    BUILDING_TYPE = "building_type"  # :"建筑类型",
    OPEN_TIME = "open_time"  # : "开盘时间",
    COMPLETE_TIME = "complete_time"  # : "竣工时间",
    DELIVERY_TIME = "delivery_time"  # : "交房时间",
    PLOT_RATIO = "plot_ratio"  # :"容积率",
    GREEN_RATIO = "green_ratio"  # :"绿化率",
    DEVELOPER = "developer"  # :"开发商名称",
    PROPERTY_COMPANY = "property_company"  # :"物业公司",
    REFER_PRICE = "refer_price"  # :"参考价",
    BUILDING_NO = "building_no"  # :"楼号/幢号",
    BUILDING_RESIDENCE_NUM = "building_residence_num"  # :"本幢楼住宅套数",
    DECORATE_TYPE = "decorate_type"  # :"装修类型",
    UNIT_NO = "unit_no"  # :"单元号",
    PROPERTY_NATURE = "property_nature"  # :"物业性质",
    FLOOR = "floor"  # :"楼层",
    HOUSE_NO = "house_no"  # :"房号",
    HOUSE_TYPE = "house_type"  # :"房型",
    HOUSE_AREA = "house_area"  # :"房屋建筑面积",
    INNER_AREA = "inner_area"  # :"套内面积",
    UNIT_PRICE = "unit_price"  # :"单价",
    SALE_STATUS = "sale_status"  # ："销售状态"
    start_urls = ["http://fcj.luan.gov.cn/ShowBuild.aspx"]

    def parse(self, response):
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        project_to_id = dict()
        projects_tag = bs_obj.find_all("a", attrs={"href": re.compile(u"ShowSection")})
        self.logger.debug("首页解析全部的楼盘对应链接:%s" % response.url)
        for project_tag in projects_tag:
            key = project_tag.get_text()
            value = urljoin(response.url, project_tag.attrs['href'])
            self.logger.debug("楼盘名称:%s, 楼盘链接:%s " % (key, value))
            project_to_id[key] = value
        meta[self.PROJECT_TO_URL] = project_to_id
        presale_permit_url = "http://fcj.luan.gov.cn/Web/PreSellInfo/ShowPreSellCertList.aspx"
        yield Request(presale_permit_url, callback=self.parse_presale_permit, meta=meta, dont_filter=True)
        pass

    def parse_presale_permit(self, response):
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        for row in bs_obj.find("table", class_="asptable4").find_all("tr")[1:]:
            columns = [item.get_text() for item in row.find_all("td")]
            meta[self.PROJECT] = _REPLACE_NBSP(columns[0])
            meta[self.PROJECT_URL] = urljoin(response.url, row.td.a.attrs["href"])
            meta[self.PROJECT_PRESELL_URL] = urljoin(response.url, row.find_all("td")[2].a.attrs["href"])
            yield Request(meta[self.PROJECT_URL], callback=self.parse_project, meta=meta, dont_filter=True)
        next_button_tag = bs_obj.find("a", attrs={"href": True, "disabled": None}, text={re.compile(u"下一页")})
        if next_button_tag:
            __VIEWSTATE = bs_obj.find("input", id="__VIEWSTATE").attrs["value"]
            __EVENTTARGET = "AspNetPager1"
            next_page_index = re.findall("'(.*?)'", next_button_tag.attrs["href"])[1]
            __EVENTARGUMENT = next_page_index
            AspNetPager1_input = int(next_page_index) - 1
            form_data = dict({
                "__VIEWSTATE": __VIEWSTATE,
                "__EVENTTARGET": __EVENTTARGET,
                "__EVENTARGUMENT": str(__EVENTARGUMENT),
                "AspNetPager1_input": str(AspNetPager1_input)
            })
            self.logger.info("下一页的页码为：%s" % __EVENTARGUMENT)
            yield FormRequest(response.url, callback=self.parse_presale_permit, formdata=form_data, meta=meta,
                              dont_filter=True)
        pass

    def parse_project(self, response):
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        meta[self.PROJECT] = bs_obj.find("span", id="ProjectName").get_text()
        meta[self.PROJECT_ADDR] = bs_obj.find("span", id="ProjectAddress").get_text()
        meta[self.PROPERTY_COMPANY] = bs_obj.find("span", id="EstateManageCompany").get_text()
        meta[self.PLOT_RATIO] = bs_obj.find("span", id="CubageRate").get_text()
        meta[self.GREEN_RATIO] = bs_obj.find("span", id="GreenRate").get_text()
        meta[self.DECORATE_TYPE] = bs_obj.find("span", id="DeliveStandard").get_text()

        self.logger.info("项目名称:%s, 项目地址:%s, 物业公司：%s，容积率：%s, 绿化率: %s, 装修类型: %s" %
                         (meta[self.PROJECT], meta[self.PROJECT_ADDR], meta[self.PROPERTY_COMPANY],
                          meta[self.PLOT_RATIO], meta[self.GREEN_RATIO],
                          meta[self.DECORATE_TYPE]))
        yield Request(meta[self.PROJECT_PRESELL_URL], callback=self.parse_project_presell, meta=meta, dont_filter=True)
        pass

    def parse_project_presell(self, response):
        """
        PRESALE_PERMIT = "presale_permit"  # :"预售证号码",
        DEVELOPER = "developer"  # :"开发商名称",
        OPEN_TIME = "open_time"  # : "开盘时间",
        BUILD_NO = "build_no"  # :"本期住宅楼号/幢号",
        BUILDING_RESIDENCE_NUM = "building_residence_num"  # :"本幢楼住宅套数",
        PROPERTY_TYPE = "property_type"  # :"物业类型/房屋类型",
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        meta[self.PRESALE_PERMIT] = bs_obj.find("span", id="证书编号").get_text()
        meta[self.DEVELOPER] = bs_obj.find("span", id="预售单位").get_text()
        meta[self.OPEN_TIME] = bs_obj.find("span", id="开盘日期").get_text()
        meta[self.BUILD_NO] = bs_obj.find("span", id="txtDong").get_text()
        meta[self.BUILDING_RESIDENCE_NUM] = bs_obj.find("span", id="预售套数").get_text()
        meta[self.PROPERTY_TYPE] = bs_obj.find("span", id="性质").get_text()

        self.logger.info("项目名称:%s,预售证号码:%s ,开发商名称:%s，开盘时间:%s ,本期住宅楼号/幢号:%s,"
                         "本幢楼住宅套数:%s, 物业类型/房屋类型:%s " %
                         (meta[self.PROJECT], meta[self.PRESALE_PERMIT], meta[self.DEVELOPER], meta[self.OPEN_TIME],
                          meta[self.BUILD_NO], meta[self.BUILDING_RESIDENCE_NUM], meta[self.PROPERTY_TYPE]))
        if meta[self.PROJECT] in meta[self.PROJECT_TO_URL].keys():
            project_show_url = meta[self.PROJECT_TO_URL][meta[self.PROJECT]]
            yield Request(project_show_url, callback=self.parse_project_show_url, meta=meta, dont_filter=True)
        else:
            self.logger.error("当前楼盘在页面中不存在，楼盘名称为:%s" % meta[self.PROJECT])

        pass

    def parse_project_show_url(self, response):
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        project_floor_url_info = dict()
        project_sum_info_url = urljoin(response.url, bs_obj.find("td", id="ctl00_ContentPlaceHolder1_tdProjectSum").a.
                                       attrs["href"])
        for project_building_tag in bs_obj.find("table", attrs={"border": 0, "style": True}).find_all("tr")[1:]:
            key = _REPLACE_NBSP(project_building_tag.find_all("td")[2].get_text())
            value = urljoin(response.url, project_building_tag.find_all("td")[2].a.attrs["href"])
            self.logger.info("栋号:%s,楼盘信息连接:%s" % (key, value))
            project_floor_url_info[key] = value
        meta[self.PROJECT_FLOOR_INFO_URL] = project_floor_url_info
        yield Request(project_sum_info_url, callback=self.parse_project_sum_info_url, meta=meta, dont_filter=True)
        pass

    def parse_project_sum_info_url(self, response):
        """
        BUILDING_NO = "building_no"  # :"楼号/幢号",
        RESIDENCE_SET = "residence_set"  # :"本期住宅总套数",
        RESIDENCE_AREA = "residence_area"  # :"本期住宅总面积",
        UNSOLD_RESIDENCE_SET = "unsold_residence_set"  # :"本期未售住宅总套数",
        UNSOLD_RESIDENCE_AREA = "unsold_residence_area"  # :"本期未售住宅总面积",
        SOLD_RESIDENCE_SET= "sold_residence_set  # :"本期已售住宅总套数",
        SOLD_RESIDENCE_AREA = "sold_residence_area"  # :"本期已售住宅总面积",
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        for row in bs_obj.find_all("tr", attrs={"align": None, "style": True}):
            columns = [_REPLACE_NBSP(item.get_text()) for item in row.find_all("td")]
            meta[self.BUILDING_NO] = columns[0]
            meta[self.RESIDENCE_SET] = columns[1]
            meta[self.RESIDENCE_AREA] = columns[2]
            meta[self.UNSOLD_COMMERCIAL_SET] = columns[3]
            meta[self.UNSOLD_COMMERCIAL_AREA] = columns[4]
            meta[self.SOLD_COMMERCIAL_SET] = float(columns[11]) + float(columns[13]) + float(columns[15]) + float(
                columns[17])
            meta[self.UNSOLD_COMMERCIAL_AREA] = \
                float(columns[12]) + float(columns[14]) + float(columns[16]) + float(columns[18])
            self.logger.info("项目名称:%s,楼号/幢号:%s,本期住宅总套数%s,本期住宅总面积:%s,本期未售住宅总套数:%s"
                             "本期未售住宅总面积:%s,本期已售住宅总套数:%s,本期已售住宅总面积:%s" %
                             (meta[self.PROJECT], meta[self.BUILDING_NO], meta[self.RESIDENCE_SET],
                              meta[self.RESIDENCE_AREA], meta[self.UNSOLD_COMMERCIAL_SET],
                              meta[self.UNSOLD_COMMERCIAL_AREA], meta[self.SOLD_COMMERCIAL_SET],
                              meta[self.UNSOLD_COMMERCIAL_AREA]))
            if meta[self.BUILDING_NO] in meta[self.PROJECT_FLOOR_INFO_URL].keys():
                project_floor_info_url = meta[self.PROJECT_FLOOR_INFO_URL][meta[self.BUILDING_NO]]
                yield Request(project_floor_info_url, callback=self.parse_project_floor_info_url, meta=meta,
                              dont_filter=True)
            else:
                self.logger.error("当前栋号不存在,栋号为:%s" % (meta[self.BUILDING_NO]))
        pass

    def parse_project_floor_info_url(self, response):
        """
        UNIT_NO = "unit_no"  # :"单元号",
        FLOOR = "floor"  # :"楼层",
        HOUSE_NO = "house_no"  # :"房号",
        HOUSE_TYPE = "house_type"  # :"房型",
        HOUSE_AREA = "house_area"  # :"房屋建筑面积",
        INNER_AREA = "inner_area"  # :"套内面积",
        UNIT_PRICE = "unit_price"  # :"单价",
        PROPERTY_NATURE = "property_nature"  # :"物业性质",
        SALE_STATUS = "sale_status" # ："销售状态"
        :param response:
        :return:
        """
        meta = response.meta
        bs_obj = BeautifulSoup(response.text, "html5lib")
        for row in bs_obj.find("table", id="Room1__room_map_").find_all("tr")[1:]:
            floor = row.td.get_text()
            tag = bs_obj.find("table", id="Room1__room_map_").tr
            units = [unit.get_text() for unit in tag.find_all("td")[1:]]
            if row.find_all("td", attrs={"title": True}, recursive=False):
                for index, house in enumerate(row.find_all("td", attrs={"title": True}, recursive=False)):
                    title = house.attrs["title"]
                    house_infos = [re.findall(u"：(.*)", item)[0] for item in title.split("\n")]
                    sale_status = self.color_to_status[house.attrs["bgcolor"]]
                    meta[self.UNIT_NO] = units[index]
                    meta[self.HOUSE_NO] = house_infos[0]
                    meta[self.HOUSE_TYPE] = house_infos[1]
                    meta[self.INNER_AREA] = house_infos[2]
                    meta[self.HOUSE_AREA] = house_infos[3]
                    meta[self.PROPERTY_NATURE] = house_infos[4]
                    meta[self.SALE_STATUS] = sale_status
                    meta[self.FLOOR] = floor
                    self.logger.info("项目名称:%s,楼号/ 幢号:%s,房号:%s,楼层:%s,单元号:%s,房型:%s,套内面积%s,房屋建筑面积:%s,物业性质:%s,销售状态:%s"
                                     % (meta[self.PROJECT], meta[self.BUILDING_NO], meta[self.HOUSE_NO],
                                        meta[self.FLOOR], meta[self.UNIT_NO], meta[self.HOUSE_TYPE],
                                        meta[self.INNER_AREA],
                                        meta[self.HOUSE_AREA], meta[self.PROPERTY_NATURE], meta[self.SALE_STATUS]))
        yield self.meta_to_item(meta)
        pass

    def meta_to_item(self, meta):
        item = RealestatecommitteeItem()
        for field in RealestatecommitteeItem.fields:
            if field not in meta.keys():
                item[field] = ""
            else:
                item[field] = meta[field]
        return item
